# -*- coding: utf-8 -*-
import scrapy,random
from weibo.agent import agent
import json,jsonpath
from weibo.items import WeiboItem
from weibo import settings
from weibo.fashion import uid


class SinaSpider(scrapy.Spider):
    name = 'sina'
    custom_settings = {
        "DEFAULT_REQUEST_HEADERS" : {
            'user-Agent': random.choice(agent),
            "Host":"m.weibo.cn",
            "Referer":"https://m.weibo.cn/",
        }
    }
    fans_url = 'https://m.weibo.cn/api/container/getIndex?containerid=%s'
    blog_fllow_url = 'https://m.weibo.cn/api/container/getIndex?containerid=%s'
    # 分布式启动的钥匙
    # redis_key = 'start_urls'
    uids=random.choice(uid)
    def start_requests(self):
        base_url='https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_2282158111'
        fllow_count_url='https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_2282158111'
        yield scrapy.Request(url=base_url,callback=self.parse_information,cookies=settings.COOKIE)
        yield scrapy.Request(url=fllow_count_url,callback=self.parse_information,cookies=settings.COOKIE)
    # 提取粉丝列表
    def parse_information(self,response):
        fans_list =jsonpath.jsonpath(json.loads(response.text),'$..card_group')
        if fans_list:
            for fans in fans_list[-1]:
                container=fans.get('actionlog').get('cardid')
                fans_container_id=container[:6]+'_-_fans_-_'+container[-10:]
                blog_fllow_container_id=container[:6]+'_-_followers_-_'+container[-10:]
                # 构建新的博主粉丝请求列表
                fans_url=self.fans_url % fans_container_id
                # 构建新的博主粉丝关注列表
                blog_fllow_url=self.blog_fllow_url % blog_fllow_container_id
                # print(fans_id,fans_desc,fans_name,fans_fllows_nums,fans_fllow_count,fans_url)
                yield scrapy.Request(fans_url,callback=self.fans_parse,dont_filter=True)
                yield scrapy.Request(blog_fllow_url, callback=self.fans_parse,dont_filter=True)
    # 粉丝的信息
    def fans_parse(self,response):
        fans__fllow_list = jsonpath.jsonpath(json.loads(response.text), '$..card_group')
        if fans__fllow_list:
            for fans in fans__fllow_list[-1]:
                item=WeiboItem()
                # 粉丝的id
                item['userid'] = fans.get('user').get('id')
                # 粉丝身份
                item['verified_reason'] = fans.get('desc1', '')
                # 粉丝的名字
                item['username'] = fans.get('user').get('screen_name')
                # 粉丝的粉丝数量
                item['followers_count'] = fans.get('desc2', '').replace('粉丝：', '')
                # 粉丝的粉丝关注
                item['follow_count'] = fans.get('user').get('follow_count')

                container = fans.get('actionlog').get('cardid')
                fans_container_id = container[:6] + '_-_fans_-_' + container[-10:]
                blog_fllow_container_id = container[:6] + '_-_followers_-_' + container[-10:]
                # 构建新的博主粉丝请求列表
                fans_url = self.fans_url % fans_container_id
                # 构建新的博主粉丝关注列表
                blog_fllow_url = self.blog_fllow_url % blog_fllow_container_id

                yield item

                yield scrapy.Request(fans_url,callback=self.fans_parse)
                yield scrapy.Request(blog_fllow_url,callback=self.fans_parse)




